import numpy as np
import pandas as pd
from sklearn.preprocessing import OneHotEncoder,LabelEncoder,StandardScaler
from sklearn.decomposition import TruncatedSVD,PCA
from sklearn.metrics.pairwise import cosine_similarity, pairwise_distances
from sklearn.feature_extraction.text import TfidfVectorizer

seed = 1024
np.random.seed(seed)

path = '../data/'

train = pd.read_pickle(path + "train_clean.pkl")
valid = pd.read_pickle(path + "valid_clean.pkl")
dev = pd.read_pickle(path+'dev_clean.pkl')

data_all = pd.concat([train,valid,dev])
data_all.reset_index(inplace=1,drop=1)


max_features = None
ngram_range = (1,2)
min_df = 3
print('Generate tfidf')
vect_orig = TfidfVectorizer(max_features=max_features,ngram_range=ngram_range, min_df=min_df)


corpus = []
f = 'context'
data_all[f] = data_all[f].astype(str)
corpus+=data_all[f].values.tolist()

vect_orig.fit(corpus)

tfidfs = vect_orig.transform(data_all[f].values.tolist())


train_tfidf = tfidfs[:train.shape[0]]
valid_tfidf = tfidfs[train.shape[0]:(train.shape[0]+valid.shape[0])]
dev_tfidf = tfidfs[(train.shape[0]+valid.shape[0]):]

pd.to_pickle(train_tfidf, path + 'train_%s_tfidf.pkl' % f)
pd.to_pickle(valid_tfidf,path+'valid_%s_tfidf.pkl'%f)
pd.to_pickle(dev_tfidf, path + 'dev_%s_tfidf.pkl' % f)